#!/bin/bash
# 修复TCP连接数耗尽(Too many open files)的智能运维脚本
FIX_CODE_ID="tcp-connection-exhaustion"

# 配置文件路径
LIMITS_CONF="/etc/security/limits.conf"
SYSCTL_CONF="/etc/sysctl.conf"
SYSCTL_D_DIR="/etc/sysctl.d/"

# 备份文件函数
backup_file() {
    local file="$1"
    if [ -f "$file" ]; then
        local backup="${file}.bak.$(date +%Y%m%d%H%M%S)"
        cp -a "$file" "$backup"
        echo "已备份配置文件: $backup"
    fi
}

# 显示当前TCP连接状态和系统限制
show_current_status() {
    echo "==== 当前TCP连接状态 ===="
    
    # 显示TCP连接状态统计
    echo "TCP连接状态分布:"
    netstat -ant | awk 'NR>2 {print $6}' | sort | uniq -c | sort -nr
    echo
    
    # 显示系统级文件句柄限制
    echo "系统级文件句柄限制:"
    sysctl fs.file-max
    echo "当前系统文件句柄使用量:"
    cat /proc/sys/fs/file-nr
    echo
    
    # 显示TCP相关内核参数
    echo "关键TCP内核参数:"
    sysctl net.ipv4.tcp_max_syn_backlog
    sysctl net.ipv4.tcp_max_tw_buckets
    sysctl net.ipv4.tcp_fin_timeout
    sysctl net.ipv4.tcp_tw_reuse
    sysctl net.ipv4.ip_local_port_range
    echo
    
    # 显示占用连接最多的进程
    echo "占用TCP连接最多的前10个进程:"
    netstat -antp 2>/dev/null | awk 'NR>2 && $7 !~ /0.0.0.0|::/ {split($7, a, "/"); if (a[2] != "") print a[2], a[1]}' | sort | uniq -c | sort -nr | head -n 10
    echo "=========================="
    echo
}

# 临时调整系统参数（立即生效）
apply_temporary_fixes() {
    echo "应用临时系统参数调整..."
    
    # 增加系统文件句柄限制
    local current_max=$(sysctl -n fs.file-max)
    local new_max=$(( current_max * 2 ))
    if [ $new_max -lt 2000000 ]; then
        new_max=2000000
    fi
    sysctl -w fs.file-max=$new_max
    echo "系统文件句柄限制已临时调整为: $new_max"
    
    # 调整TCP连接参数
    sysctl -w net.ipv4.tcp_max_syn_backlog=16384
    sysctl -w net.ipv4.tcp_max_tw_buckets=5000
    sysctl -w net.ipv4.tcp_fin_timeout=30
    sysctl -w net.ipv4.tcp_tw_reuse=1  # 允许TIME_WAIT状态的端口复用
    sysctl -w net.ipv4.tcp_tw_recycle=0  # 禁用时间戳回收（可能导致NAT环境问题）
    sysctl -w net.ipv4.ip_local_port_range="1024 65535"  # 扩大本地端口范围
    sysctl -w net.core.somaxconn=32768
    sysctl -w net.core.netdev_max_backlog=16384
    
    echo "TCP连接参数已临时调整"
    echo
}

# 永久调整系统配置（重启后生效）
apply_permanent_fixes() {
    echo "应用永久系统配置调整..."
    
    # 备份配置文件
    backup_file "$SYSCTL_CONF"
    
    # 配置sysctl参数
    local sysctl_file="$SYSCTL_CONF"
    if [ -d "$SYSCTL_D_DIR" ]; then
        sysctl_file="${SYSCTL_D_DIR}99-tcp-connections.conf"
    fi
    
    # 添加或修改TCP相关配置
    for param in \
        "fs.file-max=2000000" \
        "net.ipv4.tcp_max_syn_backlog=16384" \
        "net.ipv4.tcp_max_tw_buckets=5000" \
        "net.ipv4.tcp_fin_timeout=30" \
        "net.ipv4.tcp_tw_reuse=1" \
        "net.ipv4.tcp_tw_recycle=0" \
        "net.ipv4.ip_local_port_range=1024 65535" \
        "net.core.somaxconn=32768" \
        "net.core.netdev_max_backlog=16384"; do
        key=${param%=*}
        value=${param#*=}
        
        if grep -q "^$key" "$sysctl_file"; then
            sed -i "s/^$key.*/$key=$value/" "$sysctl_file"
        else
            echo "$key=$value" >> "$sysctl_file"
        fi
    done
    
    # 使配置生效
    sysctl -p "$sysctl_file" &> /dev/null
    
    # 调整进程文件句柄限制
    backup_file "$LIMITS_CONF"
    
    # 添加或修改limits.conf配置
    for limit in \
        "* soft nofile 1000000" \
        "* hard nofile 2000000" \
        "root soft nofile 1000000" \
        "root hard nofile 2000000"; do
        user=${limit% *}
        type=${limit#* }
        type=${type% *}
        value=${limit##* }
        
        if grep -q "^$user $type nofile" "$LIMITS_CONF"; then
            sed -i "s/^$user $type nofile.*/$limit/" "$LIMITS_CONF"
        else
            echo "$limit" >> "$LIMITS_CONF"
        fi
    done
    
    echo "永久配置已应用，将在系统重启后保持"
    echo
}

# 清理无效TCP连接
cleanup_tcp_connections() {
    echo "开始清理无效TCP连接..."
    
    # 统计当前TIME_WAIT连接数
    local tw_count=$(netstat -ant | awk 'NR>2 && $6 == "TIME_WAIT" {print}' | wc -l)
    echo "当前TIME_WAIT状态连接数: $tw_count"
    
    if [ $tw_count -gt 10000 ]; then
        echo "发现大量TIME_WAIT连接，尝试加速回收..."
        
        # 临时降低TIME_WAIT超时时间
        sysctl -w net.ipv4.tcp_fin_timeout=10
        
        echo "TIME_WAIT连接超时已临时调整为10秒，加速回收"
    fi
    
    # 识别并处理连接数过多的进程
    local threshold=500
    local high_conn_processes=$(netstat -antp 2>/dev/null | awk 'NR>2 && $7 !~ /0.0.0.0|::/ {split($7, a, "/"); if (a[1] != "") print a[1]}' | sort | uniq -c | sort -nr | awk -v threshold=$threshold '$1 > threshold {print $2}')
    
    if [ -n "$high_conn_processes" ]; then
        echo "发现连接数过多的进程:"
        echo "连接数  PID  进程名  建议操作"
        echo "-----------------------------------------"
        for pid in $high_conn_processes; do
            count=$(netstat -antp 2>/dev/null | awk -v pid=$pid 'NR>2 && $7 ~ pid {print}' | wc -l)
            cmd=$(ps -p $pid -o comm= 2>/dev/null)
            echo "$count  $pid  $cmd  重启进程"
        done
        
        echo -e "\n是否要重启这些进程以释放连接? (y/N) "
        read -r response
        if [ "$response" = "y" ] || [ "$response" = "Y" ]; then
            for pid in $high_conn_processes; do
                cmd=$(ps -p $pid -o comm= 2>/dev/null)
                echo "尝试重启进程 $pid ($cmd)..."
                # 尝试优雅重启
                kill -HUP $pid 2>/dev/null
                # 等待片刻检查是否已重启
                sleep 2
                if ps -p $pid >/dev/null; then
                    echo "优雅重启失败，尝试强制终止进程 $pid..."
                    kill -9 $pid 2>/dev/null
                fi
            done
        else
            echo "已跳过进程重启，请稍后手动处理"
        fi
    else
        echo "未发现连接数异常的进程"
    fi
    
    echo "TCP连接清理完成"
    echo
}

# 主逻辑
echo "开始处理TCP连接数耗尽(Too many open files)问题..."
echo

# 显示当前状态
show_current_status

# 应用临时修复
apply_temporary_fixes

# 应用永久修复
apply_permanent_fixes

# 清理无效连接
cleanup_tcp_connections

# 显示修复后的关键参数
echo "==== 修复后的关键参数 ===="
sysctl fs.file-max
sysctl net.ipv4.tcp_max_syn_backlog
sysctl net.ipv4.tcp_fin_timeout
sysctl net.ipv4.tcp_tw_reuse
echo "=========================="

echo "TCP连接数耗尽问题修复操作完成"
echo "操作总结:"
echo "1. 临时提升了系统文件句柄限制和TCP连接参数"
echo "2. 配置了永久生效的系统参数"
echo "3. 清理了无效TCP连接并处理了连接数过多的进程"
echo
echo "建议后续操作:"
echo "1. 监控系统连接数变化，确认问题是否解决"
echo "2. 检查应用程序是否正确关闭TCP连接，修复连接泄漏"
echo "3. 对于高并发服务，考虑增加服务器资源或实现连接池"
echo "4. 重启系统以确保所有永久配置生效"

exit 0
